Load libraries

library(magrittr)
library(tidyverse)
library(rvest)
library(plotly)
library(ggvis)
library(glue)
library(ggridges)

Scraping

Inspired by: https://www.analyticsvidhya.com/blog/2017/03/beginners-guide-on-web-scraping-in-r-using-rvest-with-hands-on-knowledge/

Selector Gadget (Chrome extension): http://selectorgadget.com/

get_html_pages <- function(urls) map(urls,read_html)

Civic, Corolla, Sentra, RJ, km < 80000, price < R$80000, year >= 2016, particular

olxURL <- function(brand,model,page,maxPrice=80000,maxKm=80000) glue("http://rj.olx.com.br/veiculos-e-pecas/carros/{brand}/{model}?f=p&me={maxPrice}&o={page}&pe={maxKm}&rs=34")

Construct URLs

urls <- map_chr(list(
  list("honda","civic",1),
  list("honda","civic",2),
  list("toyota","corolla",1),
  list("toyota","corolla",2),
  list("toyota","corolla",3),
  list("nissan","sentra",1),
  list("nissan","sentra",2)),
  ~do.call(olxURL,.x))
# urls <- c("http://rj.olx.com.br/veiculos-e-pecas/carros/honda/civic?f=p&me=80000&pe=80000&rs=34"
#           ,"http://rj.olx.com.br/veiculos-e-pecas/carros/toyota/corolla?f=p&me=80000&pe=80000&rs=34"
#           ,"http://rj.olx.com.br/veiculos-e-pecas/carros/nissan/sentra?f=p&me=80000&pe=80000&rs=34"
#           # 2nd page
#           ,"http://rj.olx.com.br/veiculos-e-pecas/carros/honda/civic?f=p&me=80000&o=2&pe=80000&rs=34"
#           ,"http://rj.olx.com.br/veiculos-e-pecas/carros/toyota/corolla?f=p&me=80000&o=2&pe=80000&rs=34"
#           ,"http://rj.olx.com.br/veiculos-e-pecas/carros/nissan/sentra?f=p&me=80000&o=2&pe=80000&rs=34"
#           # 3d page
#           ,"http://rj.olx.com.br/veiculos-e-pecas/carros/toyota/corolla?f=p&me=80000&o=3&pe=80000&rs=34")

Get pages

pages <- get_html_pages(urls)

Auxiliary functions

get_html_href <- function(pages,css) unlist(map(pages,~(html_nodes(.x,css) %>% html_attr("href"))))
get_html_text <- function(pages,css) {

Car description

text_titles <- get_html_text(pages,".OLXad-list-title")
text_titles %>% head
[1] "Honda Civic LXR 2016 Aut IPVA 2018 PAGO - 2016"                
[2] "Honda Civic LXR 2.0 Flexone/ Carro de Seguradora/ Leiam - 2016"
[3] "Honda Civic 2.0 LXR 2016 - Raridade (14.000 kms) - 2016"       
[4] "Honda Civic 2.0 lxr 16v flex 4p automático - 2016"             
[5] "Honda Civic 2016 Impecável (Na Garantia) - 2016"               
[6] "Honda Civic LXR 2.0 Flex 2016 - 2016"                          

Preços

convertReais <- function(s) s %>%
  str_replace_all("[^\\d]+","") %>%
  as.integer()
text_prices <- get_html_text(pages,".OLXad-list-price") %>%
  convertReais(.)
text_prices %>% head
[1] 68900 55000 72000 69900 59900 73800

Km, câmbio, motor: vindo assim: “33.000 km| Câmbio: automático| Flex”

clearMkCambioMotor <- function(s) 
  c(s[1] %>% str_replace("\\.","") %>% str_replace(" [kK]m",""),
    s[2] %>% str_replace("Câmbio: ",""),
    s[3])

decodeKmCambioMotor <- function(s) s %>%
  str_replace_all("[\\n\\t]","") %>%
  str_split("\\|") %>%
  map(~str_trim(.x,"both")) %>%
  map(~clearMkCambioMotor(.x))
text_km <- get_html_text(pages,".detail-specific") %>%
  decodeKmCambioMotor(.)


[[3]]
[1] "14000"      "automático" "Flex"      

[[4]]
[1] "67000"      "automático" "Flex"      

[[5]]
[1] "59000"      "automático" "Flex"      

[[6]]
[1] "9000"       "automático" "Flex"      

Region

text_region <- get_html_text(pages,".detail-region") %>%
  str_replace_all("[\\s\\n\\t]+"," ")
text_region %>% head
[1] "Rio de Janeiro, Grajaú - DDD 21"          "Rio de Janeiro, Inhoaíba - DDD 21"       
[3] "Macaé, Centro - DDD 22"                   "Rio de Janeiro, Anil - DDD 21"           
[5] "Rio de Janeiro, Barra da Tijuca - DDD 21" "Rio de Janeiro, Del Castilho - DDD 21"   

Montagem do DF

detectBrand <- function(s) case_when(str_detect(s,"corolla")~"corolla",
                                     str_detect(s,"sentra")~"sentra",
                                     str_detect(s,"civic")~"civic",
                                     T~NA_character_)
df <- data_frame(title=str_sub(text_titles,end=-7),
                 brand=as.factor(detectBrand(str_to_lower(title))),
                 region=str_replace(text_region," - DDD \\d\\d",""),
                 ddd=str_sub(text_region,start=-2),
                 year=text_titles %>% str_sub(start=-4) %>% as.integer() %>% as.factor(),
                 price1k=round(text_prices/1000,1),
                 km1k=round(as.integer(map_chr(text_km,1))/1000,1),
                 cambio=as.factor(map_chr(text_km,2)),

Sentiment Analysis

Recursively get car description from linked page

# this will hit the server many times, use sparingly
pages_descr <- get_html_pages(df$link)

https://github.com/hadley/rvest/issues/175

#function definition
html_text_collapse <- function(x, trim = FALSE, collapse = "\n"){
  UseMethod("html_text_collapse")
}

html_text_collapse.xml_nodeset <- function(x, trim = FALSE, collapse = "\n"){
  vapply(x, html_text_collapse.xml_node, character(1), trim = trim, collapse = collapse)
}

html_text_collapse.xml_node <- function(x, trim = FALSE, collapse = "\n"){
  paste(xml2::xml_find_all(x, ".//text()"), collapse = collapse)
}

Usando workaround do “html_text_collapse” pois html_text estava comendo os
tags:

html_nodes(pages_descr[[244]],".OLXad-description") %>% html_text_collapse %>% str_sub(end=400)

Versão do get_html_text com o workaround para inserir “” nos

get_html_text_collapse <- function(pages,css) {
  text <- unlist(map(pages,~html_text_collapse(html_nodes(.x,css))))
  text %>% str_trim("both")
}

Montando novo DF com descrcições (seguindo link do anuncio), limpeza

df_descr <- df %>% mutate(descr=unlist(get_html_text_collapse(pages_descr,".OLXad-description")),
                          model=unlist(get_html_text(pages_descr,".model")))
df_descr_clean <- df_descr %>%
  mutate(key=row_number(),
         descr=descr %>% str_to_lower() %>% str_replace_all("[^[:alnum:]/]"," ") %>%
           str_replace_all("\\s{2,}"," "),
         model=str_replace(model,"Modelo:[\\n\\t]+",""))

Reporta preço mediano e N por modelo. Parece q modelos são padronizados tipo FIPE

df_descr_clean %>%
  group_by(model) %>%
  summarize(price1k_median=median(price1k),N=n()) %>%
  arrange(desc(price1k_median))

Para cada carro (identificado por “key”), liste as palavras usadas na descrição.

df_descr_words <- df_descr_clean %>%
  select(key,descr) %>%
  mutate(word=str_split(descr,"\\s+")) %>%
  select(-descr) %>%
  unnest() %>%
  filter(str_length(word)>2,
         !str_detect(word,"\\d"))

Estudo frequencial das palavras

df_descr_words_counted <- df_descr_words %>%
  count(word,sort=T) %T>% print

Salvamos em excel para manualmente marcarmos palavras com sentimento negativo

df_descr_words_counted %>%
  as.data.frame() %>%
  xlsx::write.xlsx("df_descr_words_counted.xls", row.names=F)

Le arquivo com palavras com sentimento negativo

df_tretas <- read_csv("suspicious words.csv") %T>% print

Acha anuncios q contem palavra negativa

df_descr_words_negative <- df_descr_words %>%
  semi_join(df_tretas,by="word") %>%
  group_by(key) %>%
  summarize(negativity_count=n(),
            bad_words=paste0(word,collapse=",")) %T>% 
  print

Expande df de anúncios com análise de sentimento

df_descr_clean_sentiment <- df_descr_clean %>%
  left_join(df_descr_words_negative,by="key") %>%
  mutate(negativity_count=if_else(is.na(negativity_count),0L,negativity_count)) %>%
  arrange(desc(negativity_count)) %T>%
  print

Plots

df_filt <- df_descr_clean_sentiment %>%
  filter(price1k>40,!is.na(brand),year==2016) %>%
df_filt %>%
  ggplot(aes(x=negativity_count)) +
  geom_bar(stat="count")

fun_length_y <- function(x) data.frame(y=median(x),label= paste0("N=", length(x)))
df_filt %>%
  mutate(brand=fct_reorder(brand,price1k,.desc=T),
         suspicious=negativity_count>1) %>%
  arrange(suspicious) %>% # so true is drawn last
  #mutate(suspicious=as.factor(suspicious) %>% fct_inorder() %>% fct_rev()) %>%
  ggplot(aes(x=brand,y=price1k,fill=brand)) +
  #geom_violin(alpha=.25) +

  #theme(legend.position = "none")
df_filt_medians <- df_filt %>%
  group_by(brand) %>%
  summarize(N=n(),
            price1k_median=median(price1k),
            price1k_mean=mean(price1k)) %>%
  arrange(desc(price1k_median)) %>%
  mutate(y=row_number())
  
df_filt %>%
  mutate(brand=fct_reorder(brand,price1k,.desc=T)) %>%
  ggplot(aes(price1k, brand,fill=brand)) +
  geom_density_ridges() +
  geom_text(aes(x=90,y=y+.2,label=sprintf("N=%d",N)),
            data=df_filt_medians) +
  geom_segment(aes(x=price1k_median,y=y-.1,xend=price1k_median,yend=y+.1),
            data=df_filt_medians,color="blue",size=2) +
  geom_text(aes(x=price1k_median,y=y+.2,label=sprintf("med=%.1f",price1k_median)),
            data=df_filt_medians,color="blue") +
  geom_segment(aes(x=price1k_mean,y=y-.1,xend=price1k_mean,yend=y+.1),
               data=df_filt_medians,color="red",size=2) +
  geom_text(aes(x=price1k_mean,y=y-.2,label=sprintf("avg=%.1f",price1k_mean)),
            data=df_filt_medians,color="red") +
  theme_ridges()

Plota preço vs km

df_filt %>%
  mutate(suspicious=negativity_count>4) %>%
  ggplot(aes(km1k,price1k,color=brand,group=brand)) +
  geom_point(aes(shape=suspicious,size=suspicious)) +
  geom_smooth() +
  theme_bw() +
  ggtitle("Civic, Corolla, Sentra, RJ, particular",
          subtitle="km < 80k, price < R$80k, year >= 2016")

attempt: ggvis (tooltips seems buggy, cannot yet add title)

df_filt %>%
  ggvis(~km1k, ~price1k, fill=~brand) %>%
  group_by(brand) %>%
  layer_points() %>%
  # hangs
  # add_tooltip(function(df) "hello",on="click") %>%
  layer_smooths(stroke=~brand) 

Using plotly

df_filt %>%
  group_by(brand) %>%
  mutate(fit = fitted(loess(price1k ~ km1k))) %>%
  plot_ly(x = ~km1k, text=~tooltip) %>%
  #plot_ly(x = ~km1k, y = ~price1k, group=~brand,
  # Hover text:
  #text = ~paste(title), color = ~brand, marker=list(size=10)) %>%
  add_markers(y = ~price1k, color = ~brand, marker=list(size=10)) %>%
  #add_lines(name = "spline", line = list(shape = "spline")) %>%
  add_lines(y = ~fit, color = ~brand, showlegend=F) %>%
  layout(xaxis = list(showline=T))#%>%

  # add_ribbons(data=broom::augment(m),
  #             ymin = ~.fitted - 1.96 * .se.fit,
  #             ymax = ~.fitted + 1.96 * .se.fit,
  #             line = list(color = ~brand),
  #             fillcolor = ~brand)
LS0tDQp0aXRsZTogIlIgTm90ZWJvb2siDQpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sNCi0tLQ0KDQojIExvYWQgbGlicmFyaWVzDQoNCmBgYHtyIHNldHVwfQ0KbGlicmFyeShtYWdyaXR0cikNCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeShydmVzdCkNCmxpYnJhcnkocGxvdGx5KQ0KbGlicmFyeShnZ3ZpcykNCmxpYnJhcnkoZ2x1ZSkNCmxpYnJhcnkoZ2dyaWRnZXMpDQpgYGANCg0KIyBTY3JhcGluZw0KDQpJbnNwaXJlZCBieTogaHR0cHM6Ly93d3cuYW5hbHl0aWNzdmlkaHlhLmNvbS9ibG9nLzIwMTcvMDMvYmVnaW5uZXJzLWd1aWRlLW9uLXdlYi1zY3JhcGluZy1pbi1yLXVzaW5nLXJ2ZXN0LXdpdGgtaGFuZHMtb24ta25vd2xlZGdlLw0KDQpTZWxlY3RvciBHYWRnZXQgKENocm9tZSBleHRlbnNpb24pOiBodHRwOi8vc2VsZWN0b3JnYWRnZXQuY29tLw0KDQpgYGB7cn0NCmdldF9odG1sX3BhZ2VzIDwtIGZ1bmN0aW9uKHVybHMpIG1hcCh1cmxzLHJlYWRfaHRtbCkNCmBgYA0KDQpDaXZpYywgQ29yb2xsYSwgU2VudHJhLCBSSiwga20gPCA4MDAwMCwgcHJpY2UgPCBSJDgwMDAwLCB5ZWFyID49IDIwMTYsIHBhcnRpY3VsYXINCg0KYGBge3J9DQpvbHhVUkwgPC0gZnVuY3Rpb24oYnJhbmQsbW9kZWwscGFnZSxtYXhQcmljZT04MDAwMCxtYXhLbT04MDAwMCkgZ2x1ZSgiaHR0cDovL3JqLm9seC5jb20uYnIvdmVpY3Vsb3MtZS1wZWNhcy9jYXJyb3Mve2JyYW5kfS97bW9kZWx9P2Y9cCZtZT17bWF4UHJpY2V9Jm89e3BhZ2V9JnBlPXttYXhLbX0mcnM9MzQiKQ0KYGBgDQoNCkNvbnN0cnVjdCBVUkxzDQoNCmBgYHtyfQ0KdXJscyA8LSBtYXBfY2hyKGxpc3QoDQogIGxpc3QoImhvbmRhIiwiY2l2aWMiLDEpLA0KICBsaXN0KCJob25kYSIsImNpdmljIiwyKSwNCiAgbGlzdCgidG95b3RhIiwiY29yb2xsYSIsMSksDQogIGxpc3QoInRveW90YSIsImNvcm9sbGEiLDIpLA0KICBsaXN0KCJ0b3lvdGEiLCJjb3JvbGxhIiwzKSwNCiAgbGlzdCgibmlzc2FuIiwic2VudHJhIiwxKSwNCiAgbGlzdCgibmlzc2FuIiwic2VudHJhIiwyKSksDQogIH5kby5jYWxsKG9seFVSTCwueCkpDQoNCiMgdXJscyA8LSBjKCJodHRwOi8vcmoub2x4LmNvbS5ici92ZWljdWxvcy1lLXBlY2FzL2NhcnJvcy9ob25kYS9jaXZpYz9mPXAmbWU9ODAwMDAmcGU9ODAwMDAmcnM9MzQiDQojICAgICAgICAgICAsImh0dHA6Ly9yai5vbHguY29tLmJyL3ZlaWN1bG9zLWUtcGVjYXMvY2Fycm9zL3RveW90YS9jb3JvbGxhP2Y9cCZtZT04MDAwMCZwZT04MDAwMCZycz0zNCINCiMgICAgICAgICAgICwiaHR0cDovL3JqLm9seC5jb20uYnIvdmVpY3Vsb3MtZS1wZWNhcy9jYXJyb3Mvbmlzc2FuL3NlbnRyYT9mPXAmbWU9ODAwMDAmcGU9ODAwMDAmcnM9MzQiDQojICAgICAgICAgICAjIDJuZCBwYWdlDQojICAgICAgICAgICAsImh0dHA6Ly9yai5vbHguY29tLmJyL3ZlaWN1bG9zLWUtcGVjYXMvY2Fycm9zL2hvbmRhL2NpdmljP2Y9cCZtZT04MDAwMCZvPTImcGU9ODAwMDAmcnM9MzQiDQojICAgICAgICAgICAsImh0dHA6Ly9yai5vbHguY29tLmJyL3ZlaWN1bG9zLWUtcGVjYXMvY2Fycm9zL3RveW90YS9jb3JvbGxhP2Y9cCZtZT04MDAwMCZvPTImcGU9ODAwMDAmcnM9MzQiDQojICAgICAgICAgICAsImh0dHA6Ly9yai5vbHguY29tLmJyL3ZlaWN1bG9zLWUtcGVjYXMvY2Fycm9zL25pc3Nhbi9zZW50cmE/Zj1wJm1lPTgwMDAwJm89MiZwZT04MDAwMCZycz0zNCINCiMgICAgICAgICAgICMgM2QgcGFnZQ0KIyAgICAgICAgICAgLCJodHRwOi8vcmoub2x4LmNvbS5ici92ZWljdWxvcy1lLXBlY2FzL2NhcnJvcy90b3lvdGEvY29yb2xsYT9mPXAmbWU9ODAwMDAmbz0zJnBlPTgwMDAwJnJzPTM0IikNCmBgYA0KDQpHZXQgcGFnZXMNCg0KYGBge3J9DQpwYWdlcyA8LSBnZXRfaHRtbF9wYWdlcyh1cmxzKQ0KYGBgDQoNCkF1eGlsaWFyeSBmdW5jdGlvbnMNCg0KYGBge3J9DQpnZXRfaHRtbF9ocmVmIDwtIGZ1bmN0aW9uKHBhZ2VzLGNzcykgdW5saXN0KG1hcChwYWdlcyx+KGh0bWxfbm9kZXMoLngsY3NzKSAlPiUgaHRtbF9hdHRyKCJocmVmIikpKSkNCg0KZ2V0X2h0bWxfdGV4dCA8LSBmdW5jdGlvbihwYWdlcyxjc3MpIHsNCiAgdGV4dCA8LSB1bmxpc3QobWFwKHBhZ2VzLH5odG1sX3RleHQoaHRtbF9ub2RlcygueCxjc3MpKSkpDQogIHRleHQgJT4lIHN0cl90cmltKCJib3RoIikNCn0NCmBgYA0KDQojIyMgTGlua3MNCg0KYGBge3J9DQp0ZXh0X2xpbmtzIDwtIGdldF9odG1sX2hyZWYocGFnZXMsIi5PTFhhZC1saXN0LWxpbmsiKQ0KdGV4dF9saW5rcyAlPiUgaGVhZA0KYGBgDQoNCiMjIyBDYXIgZGVzY3JpcHRpb24NCg0KYGBge3J9DQp0ZXh0X3RpdGxlcyA8LSBnZXRfaHRtbF90ZXh0KHBhZ2VzLCIuT0xYYWQtbGlzdC10aXRsZSIpDQp0ZXh0X3RpdGxlcyAlPiUgaGVhZA0KYGBgDQoNCiMjIFByZcOnb3MNCg0KYGBge3J9DQpjb252ZXJ0UmVhaXMgPC0gZnVuY3Rpb24ocykgcyAlPiUNCiAgc3RyX3JlcGxhY2VfYWxsKCJbXlxcZF0rIiwiIikgJT4lDQogIGFzLmludGVnZXIoKQ0KYGBgDQoNCmBgYHtyfQ0KdGV4dF9wcmljZXMgPC0gZ2V0X2h0bWxfdGV4dChwYWdlcywiLk9MWGFkLWxpc3QtcHJpY2UiKSAlPiUNCiAgY29udmVydFJlYWlzKC4pDQp0ZXh0X3ByaWNlcyAlPiUgaGVhZA0KYGBgDQoNCiMjIyBLbSwgY8OibWJpbywgbW90b3I6IHZpbmRvIGFzc2ltOiAiMzMuMDAwIGttXG5cdFx0IHwgQ8OibWJpbzogYXV0b23DoXRpY29cblx0XHQgfCBGbGV4Ig0KDQpgYGB7cn0NCmNsZWFyTWtDYW1iaW9Nb3RvciA8LSBmdW5jdGlvbihzKSANCiAgYyhzWzFdICU+JSBzdHJfcmVwbGFjZSgiXFwuIiwiIikgJT4lIHN0cl9yZXBsYWNlKCIgW2tLXW0iLCIiKSwNCiAgICBzWzJdICU+JSBzdHJfcmVwbGFjZSgiQ8OibWJpbzogIiwiIiksDQogICAgc1szXSkNCg0KZGVjb2RlS21DYW1iaW9Nb3RvciA8LSBmdW5jdGlvbihzKSBzICU+JQ0KICBzdHJfcmVwbGFjZV9hbGwoIltcXG5cXHRdIiwiIikgJT4lDQogIHN0cl9zcGxpdCgiXFx8IikgJT4lDQogIG1hcCh+c3RyX3RyaW0oLngsImJvdGgiKSkgJT4lDQogIG1hcCh+Y2xlYXJNa0NhbWJpb01vdG9yKC54KSkNCmBgYA0KDQpgYGB7cn0NCnRleHRfa20gPC0gZ2V0X2h0bWxfdGV4dChwYWdlcywiLmRldGFpbC1zcGVjaWZpYyIpICU+JQ0KICBkZWNvZGVLbUNhbWJpb01vdG9yKC4pDQp0ZXh0X2ttICU+JSBoZWFkDQpgYGANCg0KIyMjIFJlZ2lvbg0KDQpgYGB7cn0NCnRleHRfcmVnaW9uIDwtIGdldF9odG1sX3RleHQocGFnZXMsIi5kZXRhaWwtcmVnaW9uIikgJT4lDQogIHN0cl9yZXBsYWNlX2FsbCgiW1xcc1xcblxcdF0rIiwiICIpDQp0ZXh0X3JlZ2lvbiAlPiUgaGVhZA0KYGBgDQoNCg0KDQojIE1vbnRhZ2VtIGRvIERGDQoNCmBgYHtyfQ0KZGV0ZWN0QnJhbmQgPC0gZnVuY3Rpb24ocykgY2FzZV93aGVuKHN0cl9kZXRlY3QocywiY29yb2xsYSIpfiJjb3JvbGxhIiwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfZGV0ZWN0KHMsInNlbnRyYSIpfiJzZW50cmEiLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0cl9kZXRlY3QocywiY2l2aWMiKX4iY2l2aWMiLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFR+TkFfY2hhcmFjdGVyXykNCmBgYA0KDQoNCmBgYHtyfQ0KZGYgPC0gZGF0YV9mcmFtZSh0aXRsZT1zdHJfc3ViKHRleHRfdGl0bGVzLGVuZD0tNyksDQogICAgICAgICAgICAgICAgIGJyYW5kPWFzLmZhY3RvcihkZXRlY3RCcmFuZChzdHJfdG9fbG93ZXIodGl0bGUpKSksDQogICAgICAgICAgICAgICAgIHJlZ2lvbj1zdHJfcmVwbGFjZSh0ZXh0X3JlZ2lvbiwiIC0gREREIFxcZFxcZCIsIiIpLA0KICAgICAgICAgICAgICAgICBkZGQ9c3RyX3N1Yih0ZXh0X3JlZ2lvbixzdGFydD0tMiksDQogICAgICAgICAgICAgICAgIHllYXI9dGV4dF90aXRsZXMgJT4lIHN0cl9zdWIoc3RhcnQ9LTQpICU+JSBhcy5pbnRlZ2VyKCkgJT4lIGFzLmZhY3RvcigpLA0KICAgICAgICAgICAgICAgICBwcmljZTFrPXJvdW5kKHRleHRfcHJpY2VzLzEwMDAsMSksDQogICAgICAgICAgICAgICAgIGttMWs9cm91bmQoYXMuaW50ZWdlcihtYXBfY2hyKHRleHRfa20sMSkpLzEwMDAsMSksDQogICAgICAgICAgICAgICAgIGNhbWJpbz1hcy5mYWN0b3IobWFwX2Nocih0ZXh0X2ttLDIpKSwNCiAgICAgICAgICAgICAgICAgbW90b3I9YXMuZmFjdG9yKG1hcF9jaHIodGV4dF9rbSwzKSksDQogICAgICAgICAgICAgICAgIGxpbms9dGV4dF9saW5rcykgJVQ+JQ0KICBwcmludA0KYGBgDQoNCiMgU2VudGltZW50IEFuYWx5c2lzDQoNClJlY3Vyc2l2ZWx5IGdldCBjYXIgZGVzY3JpcHRpb24gZnJvbSBsaW5rZWQgcGFnZQ0KDQpgYGB7cn0NCiMgdGhpcyB3aWxsIGhpdCB0aGUgc2VydmVyIG1hbnkgdGltZXMsIHVzZSBzcGFyaW5nbHkNCnBhZ2VzX2Rlc2NyIDwtIGdldF9odG1sX3BhZ2VzKGRmJGxpbmspDQpgYGANCg0KDQpodHRwczovL2dpdGh1Yi5jb20vaGFkbGV5L3J2ZXN0L2lzc3Vlcy8xNzUNCg0KYGBge3J9DQojZnVuY3Rpb24gZGVmaW5pdGlvbg0KaHRtbF90ZXh0X2NvbGxhcHNlIDwtIGZ1bmN0aW9uKHgsIHRyaW0gPSBGQUxTRSwgY29sbGFwc2UgPSAiXG4iKXsNCiAgVXNlTWV0aG9kKCJodG1sX3RleHRfY29sbGFwc2UiKQ0KfQ0KDQpodG1sX3RleHRfY29sbGFwc2UueG1sX25vZGVzZXQgPC0gZnVuY3Rpb24oeCwgdHJpbSA9IEZBTFNFLCBjb2xsYXBzZSA9ICJcbiIpew0KICB2YXBwbHkoeCwgaHRtbF90ZXh0X2NvbGxhcHNlLnhtbF9ub2RlLCBjaGFyYWN0ZXIoMSksIHRyaW0gPSB0cmltLCBjb2xsYXBzZSA9IGNvbGxhcHNlKQ0KfQ0KDQpodG1sX3RleHRfY29sbGFwc2UueG1sX25vZGUgPC0gZnVuY3Rpb24oeCwgdHJpbSA9IEZBTFNFLCBjb2xsYXBzZSA9ICJcbiIpew0KICBwYXN0ZSh4bWwyOjp4bWxfZmluZF9hbGwoeCwgIi4vL3RleHQoKSIpLCBjb2xsYXBzZSA9IGNvbGxhcHNlKQ0KfQ0KYGBgDQoNClVzYW5kbyB3b3JrYXJvdW5kIGRvICJodG1sX3RleHRfY29sbGFwc2UiIHBvaXMgaHRtbF90ZXh0IGVzdGF2YSBjb21lbmRvIG9zIDxicj4gdGFnczoNCg0KYGBge3J9DQpodG1sX25vZGVzKHBhZ2VzX2Rlc2NyW1syNDRdXSwiLk9MWGFkLWRlc2NyaXB0aW9uIikgJT4lIGh0bWxfdGV4dF9jb2xsYXBzZSAlPiUgc3RyX3N1YihlbmQ9NDAwKQ0KYGBgDQoNClZlcnPDo28gZG8gZ2V0X2h0bWxfdGV4dCBjb20gbyB3b3JrYXJvdW5kIHBhcmEgaW5zZXJpciAiXG4iIG5vcyA8YnI+DQoNCmBgYHtyfQ0KZ2V0X2h0bWxfdGV4dF9jb2xsYXBzZSA8LSBmdW5jdGlvbihwYWdlcyxjc3MpIHsNCiAgdGV4dCA8LSB1bmxpc3QobWFwKHBhZ2VzLH5odG1sX3RleHRfY29sbGFwc2UoaHRtbF9ub2RlcygueCxjc3MpKSkpDQogIHRleHQgJT4lIHN0cl90cmltKCJib3RoIikNCn0NCmBgYA0KDQpNb250YW5kbyBub3ZvIERGIGNvbSBkZXNjcmNpw6fDtWVzIChzZWd1aW5kbyBsaW5rIGRvIGFudW5jaW8pLCBsaW1wZXphDQoNCmBgYHtyfQ0KZGZfZGVzY3IgPC0gZGYgJT4lIG11dGF0ZShkZXNjcj11bmxpc3QoZ2V0X2h0bWxfdGV4dF9jb2xsYXBzZShwYWdlc19kZXNjciwiLk9MWGFkLWRlc2NyaXB0aW9uIikpLA0KICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbD11bmxpc3QoZ2V0X2h0bWxfdGV4dChwYWdlc19kZXNjciwiLm1vZGVsIikpKQ0KZGZfZGVzY3JfY2xlYW4gPC0gZGZfZGVzY3IgJT4lDQogIG11dGF0ZShrZXk9cm93X251bWJlcigpLA0KICAgICAgICAgZGVzY3I9ZGVzY3IgJT4lIHN0cl90b19sb3dlcigpICU+JSBzdHJfcmVwbGFjZV9hbGwoIlteWzphbG51bTpdL10iLCIgIikgJT4lDQogICAgICAgICAgIHN0cl9yZXBsYWNlX2FsbCgiXFxzezIsfSIsIiAiKSwNCiAgICAgICAgIG1vZGVsPXN0cl9yZXBsYWNlKG1vZGVsLCJNb2RlbG86W1xcblxcdF0rIiwiIikpDQoNCmBgYA0KDQpSZXBvcnRhIHByZcOnbyBtZWRpYW5vIGUgTiBwb3IgbW9kZWxvLiBQYXJlY2UgcSBtb2RlbG9zIHPDo28gcGFkcm9uaXphZG9zIHRpcG8gRklQRQ0KDQpgYGB7cn0NCmRmX2Rlc2NyX2NsZWFuICU+JQ0KICBncm91cF9ieShtb2RlbCkgJT4lDQogIHN1bW1hcml6ZShwcmljZTFrX21lZGlhbj1tZWRpYW4ocHJpY2UxayksTj1uKCkpICU+JQ0KICBhcnJhbmdlKGRlc2MocHJpY2Uxa19tZWRpYW4pKQ0KYGBgDQoNClBhcmEgY2FkYSBjYXJybyAoaWRlbnRpZmljYWRvIHBvciAia2V5IiksIGxpc3RlIGFzIHBhbGF2cmFzIHVzYWRhcyBuYSBkZXNjcmnDp8Ojby4NCg0KYGBge3J9DQpkZl9kZXNjcl93b3JkcyA8LSBkZl9kZXNjcl9jbGVhbiAlPiUNCiAgc2VsZWN0KGtleSxkZXNjcikgJT4lDQogIG11dGF0ZSh3b3JkPXN0cl9zcGxpdChkZXNjciwiXFxzKyIpKSAlPiUNCiAgc2VsZWN0KC1kZXNjcikgJT4lDQogIHVubmVzdCgpICU+JQ0KICBmaWx0ZXIoc3RyX2xlbmd0aCh3b3JkKT4yLA0KICAgICAgICAgIXN0cl9kZXRlY3Qod29yZCwiXFxkIikpDQpgYGANCg0KRXN0dWRvIGZyZXF1ZW5jaWFsIGRhcyBwYWxhdnJhcw0KDQpgYGB7cn0NCmRmX2Rlc2NyX3dvcmRzX2NvdW50ZWQgPC0gZGZfZGVzY3Jfd29yZHMgJT4lDQogIGNvdW50KHdvcmQsc29ydD1UKSAlVD4lIHByaW50DQpgYGANCg0KU2FsdmFtb3MgZW0gZXhjZWwgcGFyYSBtYW51YWxtZW50ZSBtYXJjYXJtb3MgcGFsYXZyYXMgY29tIHNlbnRpbWVudG8gbmVnYXRpdm8NCg0KYGBge3J9DQpkZl9kZXNjcl93b3Jkc19jb3VudGVkICU+JQ0KICBhcy5kYXRhLmZyYW1lKCkgJT4lDQogIHhsc3g6OndyaXRlLnhsc3goImRmX2Rlc2NyX3dvcmRzX2NvdW50ZWQueGxzIiwgcm93Lm5hbWVzPUYpDQpgYGANCg0KTGUgYXJxdWl2byBjb20gcGFsYXZyYXMgY29tIHNlbnRpbWVudG8gbmVnYXRpdm8NCg0KYGBge3J9DQpkZl90cmV0YXMgPC0gcmVhZF9jc3YoInN1c3BpY2lvdXMgd29yZHMuY3N2IikgJVQ+JSBwcmludA0KYGBgDQoNCkFjaGEgYW51bmNpb3MgcSBjb250ZW0gcGFsYXZyYSBuZWdhdGl2YQ0KDQpgYGB7cn0NCmRmX2Rlc2NyX3dvcmRzX25lZ2F0aXZlIDwtIGRmX2Rlc2NyX3dvcmRzICU+JQ0KICBzZW1pX2pvaW4oZGZfdHJldGFzLGJ5PSJ3b3JkIikgJT4lDQogIGdyb3VwX2J5KGtleSkgJT4lDQogIHN1bW1hcml6ZShuZWdhdGl2aXR5X2NvdW50PW4oKSwNCiAgICAgICAgICAgIGJhZF93b3Jkcz1wYXN0ZTAod29yZCxjb2xsYXBzZT0iLCIpKSAlVD4lIA0KICBwcmludA0KYGBgDQoNCkV4cGFuZGUgZGYgZGUgYW7Dum5jaW9zIGNvbSBhbsOhbGlzZSBkZSBzZW50aW1lbnRvDQoNCmBgYHtyfQ0KZGZfZGVzY3JfY2xlYW5fc2VudGltZW50IDwtIGRmX2Rlc2NyX2NsZWFuICU+JQ0KICBsZWZ0X2pvaW4oZGZfZGVzY3Jfd29yZHNfbmVnYXRpdmUsYnk9ImtleSIpICU+JQ0KICBtdXRhdGUobmVnYXRpdml0eV9jb3VudD1pZl9lbHNlKGlzLm5hKG5lZ2F0aXZpdHlfY291bnQpLDBMLG5lZ2F0aXZpdHlfY291bnQpKSAlPiUNCiAgYXJyYW5nZShkZXNjKG5lZ2F0aXZpdHlfY291bnQpKSAlVD4lDQogIHByaW50DQpgYGANCg0KDQoNCg0KIyBQbG90cw0KDQoNCmBgYHtyfQ0KZGZfZmlsdCA8LSBkZl9kZXNjcl9jbGVhbl9zZW50aW1lbnQgJT4lDQogIGZpbHRlcihwcmljZTFrPjQwLCFpcy5uYShicmFuZCkseWVhcj09MjAxNikgJT4lDQogIG11dGF0ZSh0b29sdGlwPWdsdWUoIjxhIGhyZWY9e2xpbmt9Pnt0aXRsZX1cbntyZWdpb259XG48L2E+IikpDQpgYGANCg0KYGBge3J9DQpkZl9maWx0ICU+JQ0KICBnZ3Bsb3QoYWVzKHg9bmVnYXRpdml0eV9jb3VudCkpICsNCiAgZ2VvbV9iYXIoc3RhdD0iY291bnQiKQ0KYGBgDQoNCg0KDQpgYGB7cn0NCmZ1bl9sZW5ndGhfeSA8LSBmdW5jdGlvbih4KSBkYXRhLmZyYW1lKHk9bWVkaWFuKHgpLGxhYmVsPSBwYXN0ZTAoIk49IiwgbGVuZ3RoKHgpKSkNCg0KZGZfZmlsdCAlPiUNCiAgbXV0YXRlKGJyYW5kPWZjdF9yZW9yZGVyKGJyYW5kLHByaWNlMWssLmRlc2M9VCksDQogICAgICAgICBzdXNwaWNpb3VzPW5lZ2F0aXZpdHlfY291bnQ+MSkgJT4lDQogIGFycmFuZ2Uoc3VzcGljaW91cykgJT4lICMgc28gdHJ1ZSBpcyBkcmF3biBsYXN0DQogICNtdXRhdGUoc3VzcGljaW91cz1hcy5mYWN0b3Ioc3VzcGljaW91cykgJT4lIGZjdF9pbm9yZGVyKCkgJT4lIGZjdF9yZXYoKSkgJT4lDQogIGdncGxvdChhZXMoeD1icmFuZCx5PXByaWNlMWssZmlsbD1icmFuZCkpICsNCiAgI2dlb21fdmlvbGluKGFscGhhPS4yNSkgKw0KICBnZW9tX2JveHBsb3Qobm90Y2ggPSBULCBzaG93LmxlZ2VuZCA9IEYpICsNCiAgc2NhbGVfZmlsbF9kaXNjcmV0ZShndWlkZSA9ICJub25lIikgKw0KICBzdGF0X3N1bW1hcnkoZnVuLmRhdGEgPSBmdW5fbGVuZ3RoX3ksDQogICAgICAgICAgICAgICBnZW9tID0gInRleHQiLCB2anVzdCA9IC41LGhqdXN0PTIuNSwgc2l6ZSA9IDQpICsNCiAgZ2VvbV9qaXR0ZXIoYWVzKGNvbG9yPXN1c3BpY2lvdXMsc2hhcGU9c3VzcGljaW91cyksYWxwaGE9LjUsd2lkdGg9LjEsc2l6ZT0zKSArDQogIHNjYWxlX3NoYXBlX21hbnVhbCh2YWx1ZXM9YygxNiwxNykpICsNCiAgc2NhbGVfY29sb3VyX21hbnVhbCh2YWx1ZXMgPSBjKCJncmF5IiwicmVkIikpICsNCiAgdGhlbWVfYncoKSArDQogIGdndGl0bGUoIkNpdmljLCBDb3JvbGxhLCBTZW50cmEsIFJKLCBwYXJ0aWN1bGFyIiwNCiAgICAgICAgICBzdWJ0aXRsZT0ia20gPCA4MGssIHByaWNlIDwgUiQ4MGssIHllYXIgPj0gMjAxNiIpICMgKw0KICAjdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gIm5vbmUiKQ0KYGBgDQoNCmBgYHtyfQ0KZGZfZmlsdF9tZWRpYW5zIDwtIGRmX2ZpbHQgJT4lDQogIGdyb3VwX2J5KGJyYW5kKSAlPiUNCiAgc3VtbWFyaXplKE49bigpLA0KICAgICAgICAgICAgcHJpY2Uxa19tZWRpYW49bWVkaWFuKHByaWNlMWspLA0KICAgICAgICAgICAgcHJpY2Uxa19tZWFuPW1lYW4ocHJpY2UxaykpICU+JQ0KICBhcnJhbmdlKGRlc2MocHJpY2Uxa19tZWRpYW4pKSAlPiUNCiAgbXV0YXRlKHk9cm93X251bWJlcigpKQ0KICANCg0KZGZfZmlsdCAlPiUNCiAgbXV0YXRlKGJyYW5kPWZjdF9yZW9yZGVyKGJyYW5kLHByaWNlMWssLmRlc2M9VCkpICU+JQ0KICBnZ3Bsb3QoYWVzKHByaWNlMWssIGJyYW5kLGZpbGw9YnJhbmQpKSArDQogIGdlb21fZGVuc2l0eV9yaWRnZXMoKSArDQogIGdlb21fdGV4dChhZXMoeD05MCx5PXkrLjIsbGFiZWw9c3ByaW50ZigiTj0lZCIsTikpLA0KICAgICAgICAgICAgZGF0YT1kZl9maWx0X21lZGlhbnMpICsNCiAgZ2VvbV9zZWdtZW50KGFlcyh4PXByaWNlMWtfbWVkaWFuLHk9eS0uMSx4ZW5kPXByaWNlMWtfbWVkaWFuLHllbmQ9eSsuMSksDQogICAgICAgICAgICBkYXRhPWRmX2ZpbHRfbWVkaWFucyxjb2xvcj0iYmx1ZSIsc2l6ZT0yKSArDQogIGdlb21fdGV4dChhZXMoeD1wcmljZTFrX21lZGlhbix5PXkrLjIsbGFiZWw9c3ByaW50ZigibWVkPSUuMWYiLHByaWNlMWtfbWVkaWFuKSksDQogICAgICAgICAgICBkYXRhPWRmX2ZpbHRfbWVkaWFucyxjb2xvcj0iYmx1ZSIpICsNCiAgZ2VvbV9zZWdtZW50KGFlcyh4PXByaWNlMWtfbWVhbix5PXktLjEseGVuZD1wcmljZTFrX21lYW4seWVuZD15Ky4xKSwNCiAgICAgICAgICAgICAgIGRhdGE9ZGZfZmlsdF9tZWRpYW5zLGNvbG9yPSJyZWQiLHNpemU9MikgKw0KICBnZW9tX3RleHQoYWVzKHg9cHJpY2Uxa19tZWFuLHk9eS0uMixsYWJlbD1zcHJpbnRmKCJhdmc9JS4xZiIscHJpY2Uxa19tZWFuKSksDQogICAgICAgICAgICBkYXRhPWRmX2ZpbHRfbWVkaWFucyxjb2xvcj0icmVkIikgKw0KICB0aGVtZV9yaWRnZXMoKQ0KYGBgDQoNCg0KUGxvdGEgcHJlw6dvIHZzIGttDQoNCmBgYHtyfQ0KZGZfZmlsdCAlPiUNCiAgbXV0YXRlKHN1c3BpY2lvdXM9bmVnYXRpdml0eV9jb3VudD40KSAlPiUNCiAgZ2dwbG90KGFlcyhrbTFrLHByaWNlMWssY29sb3I9YnJhbmQsZ3JvdXA9YnJhbmQpKSArDQogIGdlb21fcG9pbnQoYWVzKHNoYXBlPXN1c3BpY2lvdXMsc2l6ZT1zdXNwaWNpb3VzKSkgKw0KICBnZW9tX3Ntb290aCgpICsNCiAgdGhlbWVfYncoKSArDQogIGdndGl0bGUoIkNpdmljLCBDb3JvbGxhLCBTZW50cmEsIFJKLCBwYXJ0aWN1bGFyIiwNCiAgICAgICAgICBzdWJ0aXRsZT0ia20gPCA4MGssIHByaWNlIDwgUiQ4MGssIHllYXIgPj0gMjAxNiIpDQpgYGANCg0KYXR0ZW1wdDogZ2d2aXMgKHRvb2x0aXBzIHNlZW1zIGJ1Z2d5LCBjYW5ub3QgeWV0IGFkZCB0aXRsZSkNCg0KYGBge3J9DQpkZl9maWx0ICU+JQ0KICBnZ3Zpcyh+a20xaywgfnByaWNlMWssIGZpbGw9fmJyYW5kKSAlPiUNCiAgZ3JvdXBfYnkoYnJhbmQpICU+JQ0KICBsYXllcl9wb2ludHMoKSAlPiUNCiAgIyBoYW5ncw0KICAjIGFkZF90b29sdGlwKGZ1bmN0aW9uKGRmKSAiaGVsbG8iLG9uPSJjbGljayIpICU+JQ0KICBsYXllcl9zbW9vdGhzKHN0cm9rZT1+YnJhbmQpIA0KYGBgDQoNClVzaW5nIHBsb3RseQ0KDQpgYGB7cn0NCmRmX2ZpbHQgJT4lDQogIGdyb3VwX2J5KGJyYW5kKSAlPiUNCiAgbXV0YXRlKGZpdCA9IGZpdHRlZChsb2VzcyhwcmljZTFrIH4ga20xaykpKSAlPiUNCiAgcGxvdF9seSh4ID0gfmttMWssIHRleHQ9fnRvb2x0aXApICU+JQ0KICAjcGxvdF9seSh4ID0gfmttMWssIHkgPSB+cHJpY2UxaywgZ3JvdXA9fmJyYW5kLA0KICAjIEhvdmVyIHRleHQ6DQogICN0ZXh0ID0gfnBhc3RlKHRpdGxlKSwgY29sb3IgPSB+YnJhbmQsIG1hcmtlcj1saXN0KHNpemU9MTApKSAlPiUNCiAgYWRkX21hcmtlcnMoeSA9IH5wcmljZTFrLCBjb2xvciA9IH5icmFuZCwgbWFya2VyPWxpc3Qoc2l6ZT0xMCkpICU+JQ0KICAjYWRkX2xpbmVzKG5hbWUgPSAic3BsaW5lIiwgbGluZSA9IGxpc3Qoc2hhcGUgPSAic3BsaW5lIikpICU+JQ0KICBhZGRfbGluZXMoeSA9IH5maXQsIGNvbG9yID0gfmJyYW5kLCBzaG93bGVnZW5kPUYpICU+JQ0KICBsYXlvdXQoeGF4aXMgPSBsaXN0KHNob3dsaW5lPVQpKSMlPiUNCiAgIyBhZGRfcmliYm9ucyhkYXRhPWJyb29tOjphdWdtZW50KG0pLA0KICAjICAgICAgICAgICAgIHltaW4gPSB+LmZpdHRlZCAtIDEuOTYgKiAuc2UuZml0LA0KICAjICAgICAgICAgICAgIHltYXggPSB+LmZpdHRlZCArIDEuOTYgKiAuc2UuZml0LA0KICAjICAgICAgICAgICAgIGxpbmUgPSBsaXN0KGNvbG9yID0gfmJyYW5kKSwNCiAgIyAgICAgICAgICAgICBmaWxsY29sb3IgPSB+YnJhbmQpDQpgYGANCg0KDQoNCg==